wip: Add `cargo-bench`, which runs benchmarks at --opt-level=3

diff --git a/Cargo.toml b/Cargo.toml
index e1cd0845ba4fd3c51927323b42508f78e520ed1a..4c17e2236373bfe2c796f5e9ecc9b57aae4a6a10 100644 (file)
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -60,6 +60,10 @@ test = false
 name = "cargo-test"
 test = false
 
+[[bin]]
+name = "cargo-bench"
+test = false
+
 [[bin]]
 name = "cargo-verify-project"
 test = false

diff --git a/src/bin/cargo-bench.rs b/src/bin/cargo-bench.rs
new file mode 100644 (file)
index 0000000..e700c13
--- /dev/null
+++ b/src/bin/cargo-bench.rs
@@ -0,0 +1,64 @@
+#![feature(phase)]
+
+extern crate serialize;
+extern crate cargo;
+extern crate docopt;
+#[phase(plugin)] extern crate docopt_macros;
+
+use std::io::process::ExitStatus;
+
+use cargo::ops;
+use cargo::execute_main_without_stdin;
+use cargo::core::MultiShell;
+use cargo::util::{CliResult, CliError, CargoError};
+use cargo::util::important_paths::{find_root_manifest_for_cwd};
+
+docopt!(Options, "
+Execute all benchmarks of a local package
+
+Usage:
+    cargo-bench [options] [--] [<args>...]
+
+Options:
+    -h, --help              Print this message
+    -j N, --jobs N          The number of jobs to run in parallel
+    --manifest-path PATH    Path to the manifest to build benchmarks for
+    -v, --verbose           Use verbose output
+
+All of the trailing arguments are passed to the benchmark binaries generated
+for filtering benchmarks and generally providing options configuring how they
+run.
+",  flag_jobs: Option<uint>, flag_target: Option<String>,
+    flag_manifest_path: Option<String>)
+
+fn main() {
+    execute_main_without_stdin(execute, true);
+}
+
+fn execute(options: Options, shell: &mut MultiShell) -> CliResult<Option<()>> {
+    let root = try!(find_root_manifest_for_cwd(options.flag_manifest_path));
+    shell.set_verbose(options.flag_verbose);
+
+    let mut compile_opts = ops::CompileOptions {
+        update: false,
+        env: "bench",
+        shell: shell,
+        jobs: options.flag_jobs,
+        target: None,
+        dev_deps: true,
+    };
+
+    let err = try!(ops::run_benches(&root, &mut compile_opts,
+                                    options.arg_args.as_slice()).map_err(|err| {
+        CliError::from_boxed(err, 101)
+    }));
+    match err {
+        None => Ok(None),
+        Some(err) => {
+            Err(match err.exit {
+                Some(ExitStatus(i)) => CliError::new("", i as uint),
+                _ => CliError::from_boxed(err.mark_human(), 101)
+            })
+        }
+    }
+}

diff --git a/src/bin/cargo.rs b/src/bin/cargo.rs
index ac2a3e06f604ea38645447b2cb1d3a3c41eaa340..95f43e43952476461b2426c8f91773302da53ffa 100644 (file)
--- a/src/bin/cargo.rs
+++ b/src/bin/cargo.rs
@@ -46,6 +46,7 @@ Some common cargo commands are:
     new         Create a new cargo project
     run         Build and execute src/main.rs
     test        Run the tests
+    bench       Run the benchmarks
     update      Update dependencies listed in Cargo.lock
 
 See 'cargo help <command>' for more information on a specific command.

diff --git a/src/cargo/core/manifest.rs b/src/cargo/core/manifest.rs
index c733f8962cfabda15c936e1e3b98039363b5ad01..8080823b47f9fbdb968dcfb4596d505082d0a2f9 100644 (file)
--- a/src/cargo/core/manifest.rs
+++ b/src/cargo/core/manifest.rs
@@ -407,6 +407,17 @@ impl Target {
         }
     }
 
+    pub fn bench_target(name: &str, src_path: &Path,
+                        profile: &Profile, metadata: Metadata) -> Target {
+        Target {
+            kind: BinTarget,
+            name: name.to_string(),
+            src_path: src_path.clone(),
+            profile: profile.clone(),
+            metadata: Some(metadata),
+        }
+    }
+
     pub fn get_name(&self) -> &str {
         self.name.as_slice()
     }

diff --git a/src/cargo/ops/cargo_rustc/context.rs b/src/cargo/ops/cargo_rustc/context.rs
index 5f90d92d4914282cb8e0e34d1350150203a9f52b..87934fa64960d03d2dbc0584ea62cd49e2ca21bd 100644 (file)
--- a/src/cargo/ops/cargo_rustc/context.rs
+++ b/src/cargo/ops/cargo_rustc/context.rs
@@ -254,7 +254,7 @@ impl<'a, 'b> Context<'a, 'b> {
 
     pub fn is_relevant_target(&self, target: &Target) -> bool {
         target.is_lib() && match self.env {
-            "doc" | "test" => target.get_profile().is_compile(),
+            "doc" | "test" | "bench" => target.get_profile().is_compile(),
             // doc-all == document everything, so look for doc targets and
             //            compile targets in dependencies
             "doc-all" => target.get_profile().is_compile() ||

diff --git a/src/cargo/ops/cargo_test.rs b/src/cargo/ops/cargo_test.rs
index e4c9bb7d3833aa7263d93f3f914aa82c8fa1fc7d..31e82700a32c856aeb9c2a92152246ea7adb8a2f 100644 (file)
--- a/src/cargo/ops/cargo_test.rs
+++ b/src/cargo/ops/cargo_test.rs
@@ -75,3 +75,12 @@ pub fn run_tests(manifest_path: &Path,
 
     Ok(None)
 }
+
+pub fn run_benches(manifest_path: &Path,
+                   options: &mut ops::CompileOptions,
+                   args: &[String]) -> CargoResult<Option<ProcessError>> {
+    let mut args = args.to_vec();
+    args.push("--bench".to_string());
+
+    run_tests(manifest_path, options, args.as_slice())
+}

diff --git a/src/cargo/ops/mod.rs b/src/cargo/ops/mod.rs
index db47aadaf4a837e75216147147b8ca30275920f7..16eb105e93dd2088254b2fd2996c9381be8229d3 100644 (file)
--- a/src/cargo/ops/mod.rs
+++ b/src/cargo/ops/mod.rs
@@ -7,7 +7,7 @@ pub use self::cargo_new::{new, NewOptions};
 pub use self::cargo_doc::{doc, DocOptions};
 pub use self::cargo_generate_lockfile::{generate_lockfile, write_resolve};
 pub use self::cargo_generate_lockfile::{update_lockfile, load_lockfile};
-pub use self::cargo_test::run_tests;
+pub use self::cargo_test::{run_tests, run_benches};
 
 mod cargo_clean;
 mod cargo_compile;

diff --git a/src/cargo/util/toml.rs b/src/cargo/util/toml.rs
index 20aba7688c047fd0f58cb32e17b24742306a5018..0cbea63eb817f21e35380c1cb765d0643ed261fd 100644 (file)
--- a/src/cargo/util/toml.rs
+++ b/src/cargo/util/toml.rs
@@ -22,6 +22,7 @@ pub struct Layout {
     bins: Vec<Path>,
     examples: Vec<Path>,
     tests: Vec<Path>,
+    benches: Vec<Path>,
 }
 
 impl Layout {
@@ -58,6 +59,7 @@ pub fn project_layout(root_path: &Path) -> Layout {
     let mut bins = vec!();
     let mut examples = vec!();
     let mut tests = vec!();
+    let mut benches = vec!();
 
     if root_path.join("src/lib.rs").exists() {
         lib = Some(root_path.join("src/lib.rs"));
@@ -69,6 +71,7 @@ pub fn project_layout(root_path: &Path) -> Layout {
     try_add_files(&mut examples, root_path, "examples");
 
     try_add_files(&mut tests, root_path, "tests");
+    try_add_files(&mut benches, root_path, "benches");
 
     Layout {
         root: root_path.clone(),
@@ -76,6 +79,7 @@ pub fn project_layout(root_path: &Path) -> Layout {
         bins: bins,
         examples: examples,
         tests: tests,
+        benches: benches,
     }
 }
 
@@ -157,6 +161,7 @@ type TomlLibTarget = TomlTarget;
 type TomlBinTarget = TomlTarget;
 type TomlExampleTarget = TomlTarget;
 type TomlTestTarget = TomlTarget;
+type TomlBenchTarget = TomlTarget;
 
 /*
  * TODO: Make all struct fields private
@@ -187,6 +192,7 @@ pub struct TomlManifest {
     bin: Option<Vec<TomlBinTarget>>,
     example: Option<Vec<TomlExampleTarget>>,
     test: Option<Vec<TomlTestTarget>>,
+    bench: Option<Vec<TomlTestTarget>>,
     dependencies: Option<HashMap<String, TomlDependency>>,
     dev_dependencies: Option<HashMap<String, TomlDependency>>
 }
@@ -278,6 +284,18 @@ fn inferred_test_targets(layout: &Layout) -> Vec<TomlTarget> {
     }).collect()
 }
 
+fn inferred_bench_targets(layout: &Layout) -> Vec<TomlTarget> {
+    layout.benches.iter().filter_map(|ex| {
+        ex.filestem_str().map(|name| {
+            TomlTarget {
+                name: name.to_string(),
+                path: Some(TomlPath(ex.clone())),
+                .. TomlTarget::new()
+            }
+        })
+    }).collect()
+}
+
 impl TomlManifest {
     pub fn to_manifest(&self, source_id: &SourceId, layout: &Layout)
         -> CargoResult<(Manifest, Vec<Path>)> {
@@ -340,11 +358,18 @@ impl TomlManifest {
             self.test.get_ref().iter().map(|t| t.clone()).collect()
         };
 
+        let benches = if self.bench.is_none() || self.bench.get_ref().is_empty() {
+            inferred_bench_targets(layout)
+        } else {
+            self.bench.get_ref().iter().map(|t| t.clone()).collect()
+        };
+
         // Get targets
         let targets = normalize(lib.as_slice(),
                                 bins.as_slice(),
                                 examples.as_slice(),
                                 tests.as_slice(),
+                                benches.as_slice(),
                                 &metadata);
 
         if targets.is_empty() {
@@ -443,6 +468,7 @@ struct TomlTarget {
     path: Option<TomlPath>,
     test: Option<bool>,
     doctest: Option<bool>,
+    bench: Option<bool>,
     doc: Option<bool>,
     plugin: Option<bool>,
 }
@@ -461,6 +487,7 @@ impl TomlTarget {
             path: None,
             test: None,
             doctest: None,
+            bench: None,
             doc: None,
             plugin: None,
         }
@@ -489,9 +516,10 @@ fn normalize(libs: &[TomlLibTarget],
              bins: &[TomlBinTarget],
              examples: &[TomlExampleTarget],
              tests: &[TomlTestTarget],
+             benches: &[TomlBenchTarget],
              metadata: &Metadata) -> Vec<Target> {
-    log!(4, "normalizing toml targets; lib={}; bin={}; example={}; test={}",
-         libs, bins, examples, tests);
+    log!(4, "normalizing toml targets; lib={}; bin={}; example={}; test={}, benches={}",
+         libs, bins, examples, tests, benches);
 
     enum TestDep { Needed, NotNeeded }
 
@@ -511,10 +539,16 @@ fn normalize(libs: &[TomlLibTarget],
             Some(false) => {}
         }
 
+        match target.bench {
+            Some(true) | None => ret.push(Profile::default_bench()),
+            Some(false) => {}
+        }
+
         match dep {
             Needed => {
                 ret.push(Profile::default_test().test(false));
                 ret.push(Profile::default_doc().doc(false));
+                ret.push(Profile::default_bench().test(false));
             }
             _ => {}
         }
@@ -610,9 +644,29 @@ fn normalize(libs: &[TomlLibTarget],
         }
     }
 
+    fn bench_targets(dst: &mut Vec<Target>, benches: &[TomlBenchTarget],
+                     metadata: &Metadata,
+                     default: |&TomlBenchTarget| -> String) {
+        for bench in benches.iter() {
+            let path = bench.path.clone().unwrap_or_else(|| {
+                TomlString(default(bench))
+            });
+
+            // make sure this metadata is different from any same-named libs.
+            let mut metadata = metadata.clone();
+            metadata.mix(&format!("bench-{}", bench.name));
+
+            let profile = &Profile::default_bench();
+            dst.push(Target::bench_target(bench.name.as_slice(),
+                                         &path.to_path(),
+                                         profile,
+                                         metadata));
+        }
+    }
+
     let mut ret = Vec::new();
 
-    let test_dep = if examples.len() > 0 || tests.len() > 0 {
+    let test_dep = if examples.len() > 0 || tests.len() > 0 || benches.len() > 0 {
         Needed
     } else {
         NotNeeded
@@ -646,5 +700,13 @@ fn normalize(libs: &[TomlLibTarget],
                         format!("tests/{}.rs", test.name)
                     }});
 
+    bench_targets(&mut ret, benches, metadata,
+                 |bench| {
+                     if bench.name.as_slice() == "bench" {
+                         "src/bench.rs".to_string()
+                     } else {
+                         format!("benches/{}.rs", bench.name)
+                     }});
+
     ret
 }

diff --git a/tests/test_cargo_bench.rs b/tests/test_cargo_bench.rs
new file mode 100644 (file)
index 0000000..9c2e85c
--- /dev/null
+++ b/tests/test_cargo_bench.rs
@@ -0,0 +1,825 @@
+use std::path;
+use std::str;
+
+use support::{project, execs, basic_bin_manifest, basic_lib_manifest};
+use support::{COMPILING, cargo_dir, ResultTest, FRESH, RUNNING, DOCTEST};
+use support::paths::PathExt;
+use hamcrest::{assert_that, existing_file};
+use cargo::util::process;
+
+fn setup() {}
+
+test!(cargo_bench_simple {
+    let p = project("foo")
+        .file("Cargo.toml", basic_bin_manifest("foo").as_slice())
+        .file("src/foo.rs", r#"
+            extern crate test;
+
+            fn hello() -> &'static str {
+                "hello"
+            }
+
+            pub fn main() {
+                println!("{}", hello())
+            }
+
+            #[bench]
+            fn bench_hello(_b: &mut test::Bencher) {
+                assert_eq!(hello(), "hello")
+            }"#);
+
+    assert_that(p.cargo_process("cargo-build"), execs());
+    assert_that(&p.bin("foo"), existing_file());
+
+    assert_that(
+        process(p.bin("foo")),
+        execs().with_stdout("hello\n"));
+
+    assert_that(p.process(cargo_dir().join("cargo-bench")),
+        execs().with_stdout(format!("\
+{} foo v0.5.0 ({})
+{} target[..]bench[..]foo
+
+running 1 test
+test bench_hello ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+",
+        COMPILING, p.url(),
+        RUNNING)));
+})
+
+test!(cargo_bench_verbose {
+    let p = project("foo")
+        .file("Cargo.toml", basic_bin_manifest("foo").as_slice())
+        .file("src/foo.rs", r#"
+            extern crate test;
+            fn main() {}
+            #[bench] fn bench_hello(_b: &mut test::Bencher) {}
+        "#);
+
+    assert_that(p.cargo_process("cargo-bench").arg("-v").arg("hello"),
+        execs().with_stdout(format!("\
+{running} `rustc src[..]foo.rs [..]`
+{compiling} foo v0.5.0 ({url})
+{running} `[..]target[..]bench[..]foo-[..] hello --bench`
+
+running 1 test
+test bench_hello ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+",
+        compiling = COMPILING, url = p.url(), running = RUNNING)));
+})
+
+test!(many_similar_names {
+    let p = project("foo")
+        .file("Cargo.toml", r#"
+            [package]
+            name = "foo"
+            version = "0.0.1"
+            authors = []
+        "#)
+        .file("src/lib.rs", "
+            extern crate test;
+            pub fn foo() {}
+            #[bench] fn lib_bench(_b: &mut test::Bencher) {}
+        ")
+        .file("src/main.rs", "
+            extern crate foo;
+            extern crate test;
+            fn main() {}
+            #[bench] fn bin_bench(_b: &mut test::Bencher) { foo::foo() }
+        ")
+        .file("benches/foo.rs", r#"
+            extern crate foo;
+            extern crate test;
+            #[bench] fn bench_bench(_b: &mut test::Bencher) { foo::foo() }
+        "#);
+
+    let output = p.cargo_process("cargo-bench").exec_with_output().assert();
+    let output = str::from_utf8(output.output.as_slice()).assert();
+    assert!(output.contains("test bin_bench"), "bin_bench missing\n{}", output);
+    assert!(output.contains("test lib_bench"), "lib_bench missing\n{}", output);
+    assert!(output.contains("test bench_bench"), "bench_bench missing\n{}", output);
+})
+
+test!(cargo_bench_failing_test {
+    let p = project("foo")
+        .file("Cargo.toml", basic_bin_manifest("foo").as_slice())
+        .file("src/foo.rs", r#"
+            extern crate test;
+            fn hello() -> &'static str {
+                "hello"
+            }
+
+            pub fn main() {
+                println!("{}", hello())
+            }
+
+            #[bench]
+            fn bench_hello(_b: &mut test::Bencher) {
+                assert_eq!(hello(), "nope")
+            }"#);
+
+    assert_that(p.cargo_process("cargo-build"), execs());
+    assert_that(&p.bin("foo"), existing_file());
+
+    assert_that(
+        process(p.bin("foo")),
+        execs().with_stdout("hello\n"));
+
+    assert_that(p.process(cargo_dir().join("cargo-bench")),
+        execs().with_stdout(format!("\
+{} foo v0.5.0 ({})
+{} target[..]bench[..]foo
+
+running 1 test
+test bench_hello ... ",
+        COMPILING, p.url(), RUNNING))
+              .with_stderr(format!("\
+task '<main>' failed at 'assertion failed: \
+    `(left == right) && (right == left)` (left: \
+    `hello`, right: `nope`)', src{sep}foo.rs:13
+", sep = path::SEP))
+              .with_status(101));
+})
+
+test!(bench_with_lib_dep {
+    let p = project("foo")
+        .file("Cargo.toml", r#"
+            [project]
+            name = "foo"
+            version = "0.0.1"
+            authors = []
+
+            [[bin]]
+            name = "baz"
+            path = "src/main.rs"
+        "#)
+        .file("src/lib.rs", r#"
+            extern crate test;
+            ///
+            /// ```rust
+            /// extern crate foo;
+            /// fn main() {
+            ///     println!("{}", foo::foo());
+            /// }
+            /// ```
+            ///
+            pub fn foo(){}
+            #[bench] fn lib_bench(_b: &mut test::Bencher) {}
+        "#)
+        .file("src/main.rs", "
+            extern crate foo;
+            extern crate test;
+
+            fn main() {}
+
+            #[bench]
+            fn bin_bench(_b: &mut test::Bencher) {}
+        ");
+
+    assert_that(p.cargo_process("cargo-bench"),
+        execs().with_stdout(format!("\
+{} foo v0.0.1 ({})
+{running} target[..]bench[..]baz-[..]
+
+running 1 test
+test bin_bench ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{running} target[..]bench[..]foo
+
+running 1 test
+test lib_bench ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{doctest} foo
+
+running 1 test
+test foo_0 ... ignored
+
+test result: ok. 0 passed; 0 failed; 1 ignored; 0 measured
+
+",
+        COMPILING, p.url(), running = RUNNING, doctest = DOCTEST)))
+})
+
+test!(bench_with_deep_lib_dep {
+    let p = project("bar")
+        .file("Cargo.toml", r#"
+            [package]
+            name = "bar"
+            version = "0.0.1"
+            authors = []
+
+            [dependencies.foo]
+            path = "../foo"
+        "#)
+        .file("src/lib.rs", "
+            extern crate foo;
+            extern crate test;
+            #[bench]
+            fn bar_bench(_b: &mut test::Bencher) {
+                foo::foo();
+            }
+        ");
+    let p2 = project("foo")
+        .file("Cargo.toml", r#"
+            [package]
+            name = "foo"
+            version = "0.0.1"
+            authors = []
+        "#)
+        .file("src/lib.rs", "
+            extern crate test;
+
+            pub fn foo() {}
+
+            #[bench]
+            fn foo_bench(_b: &mut test::Bencher) {}
+        ");
+
+    p2.build();
+    assert_that(p.cargo_process("cargo-bench"),
+                execs().with_status(0)
+                       .with_stdout(format!("\
+{compiling} foo v0.0.1 ({dir})
+{compiling} bar v0.0.1 ({dir})
+{running} target[..]
+
+running 1 test
+test bar_bench ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{doctest} bar
+
+running 0 tests
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured
+
+",
+                       compiling = COMPILING, running = RUNNING,
+                       doctest = DOCTEST,
+                       dir = p.url()).as_slice()));
+})
+
+test!(external_bench_explicit {
+    let p = project("foo")
+        .file("Cargo.toml", r#"
+            [project]
+            name = "foo"
+            version = "0.0.1"
+            authors = []
+
+            [[bench]]
+            name = "bench"
+            path = "src/bench.rs"
+        "#)
+        .file("src/lib.rs", r#"
+            extern crate test;
+            pub fn get_hello() -> &'static str { "Hello" }
+
+            #[bench]
+            fn internal_bench(_b: &mut test::Bencher) {}
+        "#)
+        .file("src/bench.rs", r#"
+            extern crate foo;
+            extern crate test;
+
+            #[bench]
+            fn external_bench(_b: &mut test::Bencher) {}
+        "#);
+
+    assert_that(p.cargo_process("cargo-bench"),
+        execs().with_stdout(format!("\
+{} foo v0.0.1 ({})
+{running} target[..]bench[..]bench-[..]
+
+running 1 test
+test external_bench ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{running} target[..]bench[..]foo-[..]
+
+running 1 test
+test internal_bench ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{doctest} foo
+
+running 0 tests
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured
+
+",
+        COMPILING, p.url(), running = RUNNING, doctest = DOCTEST)))
+})
+
+test!(external_bench_implicit {
+    let p = project("foo")
+        .file("Cargo.toml", r#"
+            [project]
+            name = "foo"
+            version = "0.0.1"
+            authors = []
+        "#)
+        .file("src/lib.rs", r#"
+            extern crate test;
+
+            pub fn get_hello() -> &'static str { "Hello" }
+
+            #[bench]
+            fn internal_bench(_b: &mut test::Bencher) {}
+        "#)
+        .file("benches/external.rs", r#"
+            extern crate foo;
+            extern crate test;
+
+            #[bench]
+            fn external_bench(_b: &mut test::Bencher) {}
+        "#);
+
+    assert_that(p.cargo_process("cargo-bench"),
+        execs().with_stdout(format!("\
+{} foo v0.0.1 ({})
+{running} target[..]bench[..]external-[..]
+
+running 1 test
+test external_bench ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{running} target[..]bench[..]foo-[..]
+
+running 1 test
+test internal_bench ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{doctest} foo
+
+running 0 tests
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured
+
+",
+        COMPILING, p.url(), running = RUNNING, doctest = DOCTEST)))
+})
+
+test!(dont_run_examples {
+    let p = project("foo")
+        .file("Cargo.toml", r#"
+            [project]
+            name = "foo"
+            version = "0.0.1"
+            authors = []
+        "#)
+        .file("src/lib.rs", r#"
+        "#)
+        .file("examples/dont-run-me-i-will-fail.rs", r#"
+            fn main() { fail!("Examples should not be run by 'cargo test'"); }
+        "#);
+    assert_that(p.cargo_process("cargo-bench"),
+                execs().with_status(0));
+})
+
+test!(pass_through_command_line {
+    let p = project("foo")
+        .file("Cargo.toml", r#"
+            [package]
+            name = "foo"
+            version = "0.0.1"
+            authors = []
+        "#)
+        .file("src/lib.rs", "
+            extern crate test;
+
+            #[bench] fn foo(_b: &mut test::Bencher) {}
+            #[bench] fn bar(_b: &mut test::Bencher) {}
+        ");
+
+    assert_that(p.cargo_process("cargo-bench").arg("bar"),
+                execs().with_status(0)
+                       .with_stdout(format!("\
+{compiling} foo v0.0.1 ({dir})
+{running} target[..]bench[..]foo
+
+running 1 test
+test bar ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{doctest} foo
+
+running 0 tests
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured
+
+",
+                       compiling = COMPILING, running = RUNNING,
+                       doctest = DOCTEST,
+                       dir = p.url()).as_slice()));
+
+    assert_that(p.cargo_process("cargo-bench").arg("foo"),
+                execs().with_status(0)
+                       .with_stdout(format!("\
+{compiling} foo v0.0.1 ({dir})
+{running} target[..]bench[..]foo
+
+running 1 test
+test foo ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{doctest} foo
+
+running 0 tests
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured
+
+",
+                       compiling = COMPILING, running = RUNNING,
+                       doctest = DOCTEST,
+                       dir = p.url()).as_slice()));
+})
+
+// Regression test for running cargo-bench twice with
+// tests in an rlib
+test!(cargo_bench_twice {
+    let p = project("test_twice")
+        .file("Cargo.toml", basic_lib_manifest("test_twice").as_slice())
+        .file("src/test_twice.rs", r#"
+            #![crate_type = "rlib"]
+
+            extern crate test;
+
+            #[bench]
+            fn dummy_bench(b: &mut test::Bencher) { }
+            "#);
+
+    p.cargo_process("cargo-build");
+
+    for _ in range(0u, 2) {
+        assert_that(p.process(cargo_dir().join("cargo-bench")),
+                    execs().with_status(0));
+    }
+})
+
+test!(lib_bin_same_name {
+    let p = project("foo")
+        .file("Cargo.toml", r#"
+            [project]
+            name = "foo"
+            version = "0.0.1"
+            authors = []
+
+            [[lib]]
+            name = "foo"
+            [[bin]]
+            name = "foo"
+        "#)
+        .file("src/lib.rs", "
+            extern crate test;
+            #[bench] fn lib_bench(_b: &mut test::Bencher) {}
+        ")
+        .file("src/main.rs", "
+            extern crate foo;
+            extern crate test;
+
+            #[bench]
+            fn bin_bench(_b: &mut test::Bencher) {}
+        ");
+
+    assert_that(p.cargo_process("cargo-bench"),
+        execs().with_stdout(format!("\
+{} foo v0.0.1 ({})
+{running} target[..]bench[..]foo-[..]
+
+running 1 test
+test [..] ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{running} target[..]bench[..]foo-[..]
+
+running 1 test
+test [..] ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{doctest} foo
+
+running 0 tests
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured
+
+",
+        COMPILING, p.url(), running = RUNNING, doctest = DOCTEST)))
+})
+
+test!(lib_with_standard_name {
+    let p = project("foo")
+        .file("Cargo.toml", r#"
+            [package]
+            name = "syntax"
+            version = "0.0.1"
+            authors = []
+        "#)
+        .file("src/lib.rs", "
+            extern crate test;
+
+            /// ```
+            /// syntax::foo();
+            /// ```
+            pub fn foo() {}
+
+            #[bench]
+            fn foo_bench(_b: &mut test::Bencher) {}
+        ")
+        .file("benches/bench.rs", "
+            extern crate syntax;
+            extern crate test;
+
+            #[bench]
+            fn bench(_b: &mut test::Bencher) { syntax::foo() }
+        ");
+
+    assert_that(p.cargo_process("cargo-bench"),
+                execs().with_status(0)
+                       .with_stdout(format!("\
+{compiling} syntax v0.0.1 ({dir})
+{running} target[..]bench[..]bench-[..]
+
+running 1 test
+test bench ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{running} target[..]bench[..]syntax-[..]
+
+running 1 test
+test foo_bench ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{doctest} syntax
+
+running 1 test
+test foo_0 ... ignored
+
+test result: ok. 0 passed; 0 failed; 1 ignored; 0 measured
+
+",
+                       compiling = COMPILING, running = RUNNING,
+                       doctest = DOCTEST, dir = p.url()).as_slice()));
+})
+
+test!(lib_with_standard_name2 {
+    let p = project("foo")
+        .file("Cargo.toml", r#"
+            [package]
+            name = "syntax"
+            version = "0.0.1"
+            authors = []
+
+            [[lib]]
+            name = "syntax"
+            bench = false
+            doctest = false
+        "#)
+        .file("src/lib.rs", "
+            pub fn foo() {}
+        ")
+        .file("src/main.rs", "
+            extern crate syntax;
+            extern crate test;
+
+            fn main() {}
+
+            #[bench]
+            fn bench(_b: &mut test::Bencher) { syntax::foo() }
+        ");
+
+    assert_that(p.cargo_process("cargo-bench"),
+                execs().with_status(0)
+                       .with_stdout(format!("\
+{compiling} syntax v0.0.1 ({dir})
+{running} target[..]bench[..]syntax-[..]
+
+running 1 test
+test bench ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+",
+                       compiling = COMPILING, running = RUNNING,
+                       dir = p.url()).as_slice()));
+})
+
+test!(bin_there_for_integration {
+    let p = project("foo")
+        .file("Cargo.toml", r#"
+            [package]
+            name = "foo"
+            version = "0.0.1"
+            authors = []
+        "#)
+        .file("src/main.rs", "
+            extern crate test;
+            fn main() { std::os::set_exit_status(1); }
+            #[bench] fn main_bench(_b: &mut test::Bencher) {}
+        ")
+        .file("benches/foo.rs", r#"
+            extern crate test;
+            use std::io::Command;
+            #[bench]
+            fn bench_bench(_b: &mut test::Bencher) {
+                let status = Command::new("target/bench/foo").status().unwrap();
+                assert!(status.matches_exit_status(1));
+            }
+        "#);
+
+    let output = p.cargo_process("cargo-bench").exec_with_output().assert();
+    let output = str::from_utf8(output.output.as_slice()).assert();
+    assert!(output.contains("main_bench ... bench:         0 ns/iter (+/- 0)"),
+                            "no main_bench\n{}",
+                            output);
+    assert!(output.contains("bench_bench ... bench:         0 ns/iter (+/- 0)"),
+                            "no bench_bench\n{}",
+                            output);
+})
+
+test!(bench_dylib {
+    let p = project("foo")
+        .file("Cargo.toml", r#"
+            [package]
+            name = "foo"
+            version = "0.0.1"
+            authors = []
+
+            [[lib]]
+            name = "foo"
+            crate_type = ["dylib"]
+
+            [dependencies.bar]
+            path = "bar"
+        "#)
+        .file("src/lib.rs", "
+            extern crate bar;
+            extern crate test;
+
+            pub fn bar() { bar::baz(); }
+
+            #[bench]
+            fn foo(_b: &mut test::Bencher) {}
+        ")
+        .file("benches/bench.rs", r#"
+            extern crate foo;
+            extern crate test;
+
+            #[bench]
+            fn foo(_b: &mut test::Bencher) { foo::bar(); }
+        "#)
+        .file("bar/Cargo.toml", r#"
+            [package]
+            name = "bar"
+            version = "0.0.1"
+            authors = []
+
+            [[lib]]
+            name = "bar"
+            crate_type = ["dylib"]
+        "#)
+        .file("bar/src/lib.rs", "
+             pub fn baz() {}
+        ");
+
+    assert_that(p.cargo_process("cargo-bench"),
+                execs().with_status(0)
+                       .with_stdout(format!("\
+{compiling} bar v0.0.1 ({dir})
+{compiling} foo v0.0.1 ({dir})
+{running} target[..]bench[..]bench-[..]
+
+running 1 test
+test foo ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{running} target[..]bench[..]foo-[..]
+
+running 1 test
+test foo ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{doctest} foo
+
+running 0 tests
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured
+
+",
+                       compiling = COMPILING, running = RUNNING,
+                       doctest = DOCTEST,
+                       dir = p.url()).as_slice()));
+    p.root().move_into_the_past().assert();
+    assert_that(p.process(cargo_dir().join("cargo-bench")),
+                execs().with_status(0)
+                       .with_stdout(format!("\
+{fresh} bar v0.0.1 ({dir})
+{fresh} foo v0.0.1 ({dir})
+{running} target[..]bench[..]bench-[..]
+
+running 1 test
+test foo ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{running} target[..]bench[..]foo-[..]
+
+running 1 test
+test foo ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{doctest} foo
+
+running 0 tests
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured
+
+",
+                       fresh = FRESH, running = RUNNING,
+                       doctest = DOCTEST,
+                       dir = p.url()).as_slice()));
+})
+
+test!(bench_twice_with_build_cmd {
+    let p = project("foo")
+        .file("Cargo.toml", r#"
+            [package]
+            name = "foo"
+            version = "0.0.1"
+            authors = []
+            build = 'true'
+        "#)
+        .file("src/lib.rs", "
+            extern crate test;
+            #[bench]
+            fn foo(_b: &mut test::Bencher) {}
+        ");
+
+    assert_that(p.cargo_process("cargo-bench"),
+                execs().with_status(0)
+                       .with_stdout(format!("\
+{compiling} foo v0.0.1 ({dir})
+{running} target[..]bench[..]foo-[..]
+
+running 1 test
+test foo ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{doctest} foo
+
+running 0 tests
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured
+
+",
+                       compiling = COMPILING, running = RUNNING,
+                       doctest = DOCTEST,
+                       dir = p.url()).as_slice()));
+
+    assert_that(p.process(cargo_dir().join("cargo-bench")),
+                execs().with_status(0)
+                       .with_stdout(format!("\
+{fresh} foo v0.0.1 ({dir})
+{running} target[..]bench[..]foo-[..]
+
+running 1 test
+test foo ... bench:         0 ns/iter (+/- 0)
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 1 measured
+
+{doctest} foo
+
+running 0 tests
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured
+
+",
+                       fresh = FRESH, running = RUNNING,
+                       doctest = DOCTEST,
+                       dir = p.url()).as_slice()));
+})

diff --git a/tests/tests.rs b/tests/tests.rs
index 54e8cf0b161ac2e2b6bd25b8bfaaacc40dff5e7c..966e572aa9f9084dc3b5e9e99f13a23fa7739c0d 100644 (file)
--- a/tests/tests.rs
+++ b/tests/tests.rs
@@ -22,6 +22,7 @@ macro_rules! test(
 )
 
 mod test_cargo;
+mod test_cargo_bench;
 mod test_cargo_clean;
 mod test_cargo_compile;
 mod test_cargo_compile_git_deps;